{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "be6b0e672a622c13"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-07T09:16:35.242965Z",
     "start_time": "2025-11-07T09:16:35.172482Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"数值列1\": [1, 2, np.nan, 4, 5, None],\n",
    "    \"数值列2\": [10, -999, 30, 40, -888, 60],\n",
    "    \"字符串列\": [\"a\", \"b\", \"\", \"d\", \"未知\", None],\n",
    "    \"布尔列\": [True, False, None, True, np.nan, False]\n",
    "})\n",
    "\n",
    "\n",
    "print(\"原始数据各列缺失比例：\")\n",
    "print((df.isnull().mean() * 100).round(2).astype(str) + \"%\")\n",
    "\n",
    "\n",
    "special_missing = {\n",
    "    \"数值列2\": [-999, -888],\n",
    "    \"字符串列\": [\"\", \"未知\"]\n",
    "}\n",
    "df_clean = df.copy()\n",
    "for col, vals in special_missing.items():\n",
    "    df_clean[col] = df_clean[col].replace(vals, np.nan)\n",
    "\n",
    "print(\"\\n处理特殊缺失值后各列缺失比例：\")\n",
    "print((df_clean.isnull().mean() * 100).round(2).astype(str) + \"%\")\n",
    "\n",
    "\n",
    "print(\"\\n存在缺失值的行：\")\n",
    "print(df_clean[df_clean.isnull().any(axis=1)][[\"数值列1\", \"数值列2\", \"字符串列\", \"布尔列\"]])\n"
   ],
   "id": "e4ec49269e736f10",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据各列缺失比例：\n",
      "数值列1    33.33%\n",
      "数值列2      0.0%\n",
      "字符串列    16.67%\n",
      "布尔列     33.33%\n",
      "dtype: object\n",
      "\n",
      "处理特殊缺失值后各列缺失比例：\n",
      "数值列1    33.33%\n",
      "数值列2    33.33%\n",
      "字符串列     50.0%\n",
      "布尔列     33.33%\n",
      "dtype: object\n",
      "\n",
      "存在缺失值的行：\n",
      "   数值列1  数值列2  字符串列    布尔列\n",
      "1   2.0   NaN     b  False\n",
      "2   NaN  30.0   NaN   None\n",
      "4   5.0   NaN   NaN    NaN\n",
      "5   NaN  60.0  None  False\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-07T10:16:05.575883Z",
     "start_time": "2025-11-07T10:16:05.556649Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"用户ID\": [101, 101, 102, 103, 103, 104, 105],\n",
    "    \"姓名\": [\"张三\", \"张三\", \"李四\", \"王五\", \"王五\", \"赵六\", \"孙七\"],\n",
    "    \"年龄\": [25, 25, 30, 35, 36, 28, 28],\n",
    "    \"消费金额\": [100, 100, 200, 150, 250, 300, 300],\n",
    "    \"重复列_年龄\": [25, 25, 30, 35, 36, 28, 28]\n",
    "})\n",
    "\n",
    "print(\"原始数据（含重复值）：\")\n",
    "print(df)\n",
    "\n",
    "duplicate_rows = df.duplicated()\n",
    "print(\"\\n1. 完全重复行检测（True=重复行）：\")\n",
    "print(duplicate_rows)\n",
    "\n",
    "\n",
    "df_duplicate_rows = df[duplicate_rows]\n",
    "print(f\"\\n2. 完全重复的行（共{len(df_duplicate_rows)}行）：\")\n",
    "print(df_duplicate_rows)\n",
    "total_duplicates = duplicate_rows.sum()\n",
    "print(f\"\\n3. 完全重复行总数：{total_duplicates}\")\n",
    "\n",
    "\n",
    "df_unique_full = df.drop_duplicates()\n",
    "print(f\"\\n4. 完全去重后的数据（原始{len(df)}行 → 剩余{len(df_unique_full)}行）：\")\n",
    "print(df_unique_full)"
   ],
   "id": "1cb7e09ad2df47e4",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据（含重复值）：\n",
      "   用户ID  姓名  年龄  消费金额  重复列_年龄\n",
      "0   101  张三  25   100      25\n",
      "1   101  张三  25   100      25\n",
      "2   102  李四  30   200      30\n",
      "3   103  王五  35   150      35\n",
      "4   103  王五  36   250      36\n",
      "5   104  赵六  28   300      28\n",
      "6   105  孙七  28   300      28\n",
      "\n",
      "1. 完全重复行检测（True=重复行）：\n",
      "0    False\n",
      "1     True\n",
      "2    False\n",
      "3    False\n",
      "4    False\n",
      "5    False\n",
      "6    False\n",
      "dtype: bool\n",
      "\n",
      "2. 完全重复的行（共1行）：\n",
      "   用户ID  姓名  年龄  消费金额  重复列_年龄\n",
      "1   101  张三  25   100      25\n",
      "\n",
      "3. 完全重复行总数：1\n",
      "\n",
      "4. 完全去重后的数据（原始7行 → 剩余6行）：\n",
      "   用户ID  姓名  年龄  消费金额  重复列_年龄\n",
      "0   101  张三  25   100      25\n",
      "2   102  李四  30   200      30\n",
      "3   103  王五  35   150      35\n",
      "4   103  王五  36   250      36\n",
      "5   104  赵六  28   300      28\n",
      "6   105  孙七  28   300      28\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
